# _*_ coding:utf-8 _*_
# @Time  : 2022.07.07
# @Author: zizlee
# 上期所数据爬取、解析器
import json
import time
import pathlib
import requests
import datetime
import numpy as np
import pandas as pd
from itertools import groupby
from urllib3 import disable_warnings
from zizlee_position import get_dominant_price_position, get_contract_price_position
disable_warnings()

USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) " \
             "Chrome/78.0.3904.108 Safari/537.36"

VARIETY_EN = {
    'IF': 'IF',
    'IC': 'IC',
    'IH': 'IH',
    '生猪': 'LH',
    '沪深300': 'IF',
    '中证500': 'IC',
    '上证50': 'IH',
    '铜(CU)': 'CU',
    '铜': 'CU',
    '铜(BC)': 'BC',
    '短纤': 'PF',
    '铝': 'AL',
    '铅': 'PB',
    '花生': 'PK',
    '锌': 'ZN',
    '锡': 'SN',
    '镍': 'NI',
    '铁矿石': 'I',
    '热轧卷板': 'HC',
    '热卷': 'HC',
    '螺纹钢': 'RB',
    '螺纹': 'RB',
    '螺纹钢仓库': 'RB',
    '螺纹钢厂库': 'RB',
    '线材': 'WR',
    '不锈钢': 'SS',
    '不锈钢仓库': 'SS',
    '硅铁': 'SF',
    '硅锰': 'SM',
    '锰硅': 'SM',
    '焦煤': 'JM',
    '焦炭': 'J',
    '动力煤': 'ZC',
    '郑煤': 'ZC',
    '黄金': 'AU',
    '白银': 'AG',
    '大豆': 'A',
    '豆一': 'A',
    '豆二': 'B',
    '胶合板': 'BB',
    '豆粕': 'M',
    '豆油': 'Y',
    '棕榈油': 'P',
    '粳米': 'RR',
    '白糖': 'SR',
    '棉花': 'CF',
    '棉纱': 'CY',
    '苹果': 'AP',
    '红枣': 'CJ',
    '聚丙烯': 'PP',
    '聚氯乙烯': 'V',
    '聚乙烯': 'L',
    '鸡蛋': 'JD',
    '菜粕': 'RM',
    '菜籽粕': 'RM',
    '菜油': 'OI',
    '菜籽油': 'OI',
    '玉米': 'C',
    '淀粉': 'CS',
    'LLDPE': 'L',
    'PP': 'PP',
    'PVC': 'V',
    '苯乙烯': 'EB',
    '低硫燃料油仓库': 'LU',
    '低硫燃料油厂库': 'LU',
    '全乳胶': 'RU',
    '天然橡胶': 'RU',
    '橡胶': 'RU',
    '20号胶': 'NR',
    'STR20': 'NR',
    '甲醇': 'MA',
    '尿素': 'UR',
    '玻璃': 'FG',
    '纯碱': 'SA',
    '乙二醇': 'EG',
    'PTA': 'TA',
    '纸浆': 'SP',
    '纸浆仓库': 'SP',
    '纸浆厂库': 'SP',
    '沥青': 'BU',
    '沥青仓库': 'BU',
    '沥青厂库': 'BU',
    '石油沥青仓库': 'BU',
    '石油沥青厂库': 'BU',
    '纤维板': 'FB',
    '液化气': 'PG',
    'LPG': 'PG',
    '燃料油': 'FU',
    '液化石油气': 'PG',
    '原油': 'SC',
    '玉米淀粉': 'CS',
    '中质含硫原油': 'SC',
    '氧化铝仓库': 'AO',
    '氧化铝厂库': 'AO',
    '丁二烯橡胶仓库': 'BR',
    '丁二烯橡胶厂库': 'BR'
}


def split_number_en(ustring):
    return [''.join(list(g)) for k, g in groupby(ustring, key=lambda x: x.isdigit())]


def full_width_to_half_width(ustring):
    """ 全角转半角 """
    reverse_str = ""
    for uchar in ustring:
        inside_code = ord(uchar)
        if inside_code == 12288:  # 全角空格直接转换
            inside_code = 32
        elif 65281 <= inside_code <= 65374:  # 全角字符（除空格）根据关系转化
            inside_code -= 65248
        else:
            pass
        reverse_str += chr(inside_code)
    return reverse_str


class DailyQuoteSpider(object):
    def __init__(self, date: datetime.datetime):
        self.quote_date = datetime.datetime.strptime(date.strftime('%Y-%m-%d'), '%Y-%m-%d')
        self.quote_date_string = self.quote_date.strftime('%Y%m%d')
        self.headers = {'User-Agent': USER_AGENT,
                        'Host': 'www.shfe.com.cn'}
        self.quote_url = "http://www.shfe.com.cn/data/dailydata/kx/kx{}.dat".format(self.quote_date_string)
        self.quote_file = pathlib.Path("shfe/daily/{}.json".format(self.quote_date_string))
        self.rank_url = "http://www.shfe.com.cn/data/dailydata/kx/pm{}.dat".format(self.quote_date_string)
        self.rank_file = pathlib.Path("shfe/rank/{}.json".format(self.quote_date_string))
        self.receipt_url = "http://www.shfe.com.cn/data/dailydata/{}dailystock.dat".format(self.quote_date_string)
        self.receipt_file = pathlib.Path("shfe/receipt/{}.json".format(self.quote_date_string))

    # 获取日行情文件
    def get_quote_file(self):
        r = requests.get(self.quote_url, headers=self.headers)
        with open(self.quote_file, 'w', encoding='utf8') as fp:
            json.dump(r.json(), fp=fp, ensure_ascii=False, indent=2)
        print('获取{}.SHF日行情数据完成！'.format(self.quote_date_string))

    # 获取日排名文件
    def get_rank_file(self):
        r = requests.get(self.rank_url, headers=self.headers)
        with open(self.rank_file, 'w', encoding='utf8') as fp:
            json.dump(r.json(), fp=fp, ensure_ascii=False, indent=2)
        print('获取{}.SHF日排名数据完成！'.format(self.quote_date_string))

    # 获取日仓单文件
    def get_receipt_file(self):
        r = requests.get(self.receipt_url, headers=self.headers)
        with open(self.receipt_file, 'w', encoding='utf8') as fp:
            json.dump(r.json(), fp=fp, ensure_ascii=False, indent=2)
        print('获取{}.SHF日仓单数据完成！'.format(self.quote_date_string))


class DailyQuoteParser(object):
    # SERVER_API = "http://127.0.0.1:8000/api/"
    SERVER_API = "https://210.13.218.130:9000/api/"

    def __init__(self, date: datetime.datetime):
        self.quote_date = datetime.datetime.strptime(date.strftime('%Y-%m-%d'), '%Y-%m-%d')
        self.quote_date_string = self.quote_date.strftime('%Y%m%d')

        self.quote_file = pathlib.Path("shfe/daily/{}.json".format(self.quote_date_string))
        self.rank_file = pathlib.Path("shfe/rank/{}.json".format(self.quote_date_string))
        self.receipt_file = pathlib.Path("shfe/receipt/{}.json".format(self.quote_date_string))

        self.quote_save_url = self.SERVER_API + "exchange/shfe/daily/?date={}".format(self.quote_date.strftime("%Y-%m-%d"))
        self.rank_save_url = self.SERVER_API + "exchange/shfe/rank/?date={}".format(self.quote_date.strftime("%Y-%m-%d"))
        self.receipt_save_url = self.SERVER_API + "exchange/shfe/receipt/"

        self.resolution_quote_file = pathlib.Path("resolution/{}/SHF_Quote.json".format(self.quote_date_string))
        self.resolution_rank_file = pathlib.Path("resolution/{}/SHF_Rank.json".format(self.quote_date_string))
        self.resolution_receipt_file = pathlib.Path("resolution/{}/SHF_Receipt.json".format(self.quote_date_string))
        self.resolution_receipt_file2 = pathlib.Path("resolution/{}/SHF_Receipt2.json".format(self.quote_date_string))
        resolution_folder = pathlib.Path("resolution/{}/".format(self.quote_date_string))
        if not resolution_folder.exists():
            resolution_folder.mkdir(parents=True)

    # 解析日行情数据
    def parse_quote_file(self):
        if not self.quote_file.exists():
            print("没有发现{}.SHF的行情文件,请先抓取数据!".format(self.quote_date_string))
            return
        with open(self.quote_file, "r", encoding="utf-8") as reader:
            source_content = json.load(reader)
        # 解析content转为DataFrame
        json_df = pd.DataFrame(source_content['o_curinstrument'])
        # 选取PRODUCTID非总计、DELIVERYMONTH非小计的行
        json_df = json_df[~json_df['PRODUCTID'].str.contains('总计|小计|合计|_tas|efp')]  # 选取品种不含有小计和总计合计_tas的行
        json_df = json_df[~json_df['DELIVERYMONTH'].str.contains('总计|小计|合计')]  # 选取合约不含有小计和总计合计的行
        # 处理空格
        json_df["PRODUCTID"] = json_df["PRODUCTID"].apply(lambda x: x.split("_")[0].upper())
        # json_df["PRODUCTGROUPID"] = json_df["PRODUCTGROUPID"].str.strip().str.upper()
        json_df["PRODUCTNAME"] = json_df["PRODUCTNAME"].str.strip()
        json_df["DELIVERYMONTH"] = json_df["DELIVERYMONTH"].str.strip()
        json_df = json_df.reindex(columns=["DATE", "PRODUCTID", "DELIVERYMONTH", "PRESETTLEMENTPRICE", "OPENPRICE",
                                           "HIGHESTPRICE", "LOWESTPRICE", "CLOSEPRICE", "SETTLEMENTPRICE",
                                           "ZD1_CHG", "ZD2_CHG", "VOLUME", "TURNOVER", "OPENINTEREST", "OPENINTERESTCHG"])
        json_df["DATE"] = int(self.quote_date.timestamp())
        # 修改列头，返回
        json_df.columns = ["date", "variety_en", "contract", "pre_settlement", "open_price", "highest", "lowest",
                           "close_price", "settlement", "zd_1", "zd_2", "trade_volume", "trade_price", "empty_volume",
                           "increase_volume"]
        # 合约等于品种+合约
        json_df["contract"] = json_df["variety_en"] + json_df["contract"]
        # 处理没有数据或空值=补0
        json_df.replace(to_replace="^\s*$", value=np.nan, regex=True, inplace=True)
        json_df = json_df.fillna(0)
        quote_data = json_df.to_dict(orient='records')
        print('----- 解析{}.SHF行情文件成功,数量:{} -----'.format(self.quote_date_string, len(quote_data)))
        if len(quote_data) > 1:
            # 保存到本地文件
            with open(self.resolution_quote_file, 'w', encoding='utf8') as fp:
                json.dump(quote_data, fp, indent=2, ensure_ascii=False)

    # 解析日排名数据
    def parse_rank_file(self):
        if not self.rank_file.exists():
            print("没有发现{}.SHF的排名文件,请先抓取数据!".format(self.quote_date_string))
            return
        with open(self.rank_file, "r", encoding="utf-8") as reader:
            source_content = json.load(reader)
        json_df = pd.DataFrame(source_content["o_cursor"])
        # 取排名在(1~20的数据[-1:期货公司总计,0:非期货公司总计,999品种合约总计]
        json_df = json_df[(json_df["RANK"] >= 1) & (json_df['RANK'] <= 20)]
        # 去除字符串空格
        json_df["INSTRUMENTID"] = json_df["INSTRUMENTID"].str.strip().str.upper().str.replace("ALL", '')
        json_df["PARTICIPANTABBR1"] = json_df["PARTICIPANTABBR1"].str.strip()
        json_df["PARTICIPANTABBR2"] = json_df["PARTICIPANTABBR2"].str.strip()
        json_df["PARTICIPANTABBR3"] = json_df["PARTICIPANTABBR3"].str.strip()
        # 空值处理补0
        json_df.replace(to_replace="^\s*$", value=np.nan, regex=True, inplace=True)
        json_df = json_df.fillna(0)
        # 新增品种代码列和DATE列
        json_df["VARIETYEN"] = json_df["INSTRUMENTID"].apply(split_number_en).apply(lambda x: x[0].upper())
        json_df["DATE"] = int(self.quote_date.timestamp())
        # 重新设置索引
        json_df = json_df.reindex(columns=["DATE", "VARIETYEN", "INSTRUMENTID", "RANK", "PARTICIPANTABBR1", "CJ1", "CJ1_CHG", "PARTICIPANTABBR2", "CJ2", "CJ2_CHG","PARTICIPANTABBR3", "CJ3", "CJ3_CHG"])
        json_df.columns = ["date", "variety_en", "contract", "rank",
                           "trade_company", "trade", "trade_increase",
                           "long_position_company", "long_position", "long_position_increase",
                           "short_position_company", "short_position", "short_position_increase"]

        # for i in json_df.itertuples():
        #     print(i)
        # print(json_df.shape)
        # 处理数据类型
        if not json_df.empty:
            # 转换数据类型
            json_df["rank"] = json_df["rank"].astype("int")
            json_df["trade_company"] = json_df["trade_company"].apply(lambda x: '-' if x == 0 else x)
            json_df["trade"] = json_df["trade"].astype("int")
            json_df["trade_increase"] = json_df["trade_increase"].astype("int")
            json_df["long_position_company"] = json_df["long_position_company"].apply(lambda x: '-' if x == 0 else x)
            json_df["long_position"] = json_df["long_position"].astype("int")
            json_df["long_position_increase"] = json_df["long_position_increase"].astype("int")
            json_df["short_position_company"] = json_df["short_position_company"].apply(lambda x: '-' if x == 0 else x)
            json_df["short_position"] = json_df["short_position"].astype("int")
            json_df["short_position_increase"] = json_df["short_position_increase"].astype("int")
        rank_data = json_df.to_dict(orient='records')
        print('----- 解析{}.SHF排名文件成功,数量:{} -----'.format(self.quote_date_string, len(rank_data)))
        if len(rank_data) > 1:
            # 保存到本地文件
            with open(self.resolution_rank_file, 'w', encoding='utf8') as fp:
                json.dump(rank_data, fp, indent=2, ensure_ascii=False)

    # 解析日仓单文件
    def parse_receipt_file(self):
        if not self.receipt_file.exists():
            print("没有发现{}.SHF的仓单文件,请先抓取数据!".format(self.quote_date_string))
            return
        with open(self.receipt_file, "r", encoding="utf-8") as reader:
            source_content = json.load(reader)
        json_df = pd.DataFrame(source_content['o_cursor'])
        # 处理仓库名称
        json_df["WHABBRNAME"] = json_df["WHABBRNAME"].apply(full_width_to_half_width)
        json_df["WHABBRNAME"] = json_df["WHABBRNAME"].apply(lambda name: name.split("$$")[0].strip())
        # 去掉含仓库名称含合计或小计的行
        json_df = json_df[~json_df['WHABBRNAME'].str.contains('总计|小计|合计')]  # 选取品种不含有小计和总计合计的行
        # 处理品种名称
        json_df["VARNAME"] = json_df["VARNAME"].apply(lambda name: name.split("$$")[0].strip())
        # 增加交易代码列
        json_df["VAREN"] = json_df['VARNAME'].apply(self.get_variety_en)
        # 仓单和增减转为int类型
        json_df["WRTWGHTS"] = json_df["WRTWGHTS"].apply(int)
        json_df["WRTCHANGE"] = json_df["WRTCHANGE"].apply(int)
        # 整理出想要的数据列(仓库名称(简称),交易代码,仓单,增减)
        result_df = json_df.reindex(columns=["WHABBRNAME", "VAREN", "WRTWGHTS", "WRTCHANGE"])
        result_df.columns = ["warehouse", "variety_en", "receipt", "increase"]
        # 以品种分组求和
        sum_df = result_df.groupby(by=['variety_en'], as_index=False)[['receipt', 'increase']].sum()
        sum_df["date"] = int(self.quote_date.timestamp())
        sum_df.columns = ["variety_en", "receipt", "increase", "date"]
        sum_df = sum_df.reindex(columns=["date", "variety_en", "receipt", "increase"])
        receipt_data = sum_df.to_dict(orient='records')
        print('----- 解析{}.SHF仓单文件成功,数量:{} -----'.format(self.quote_date_string, len(receipt_data)))
        if len(receipt_data) > 1:
            # 保存到本地文件
            with open(self.resolution_receipt_file, 'w', encoding='utf8') as fp:
                json.dump(receipt_data, fp, indent=2, ensure_ascii=False)

    # 解析日仓单文件2(带仓库名的数据)
    def parse_receipt_file2(self):
        if not self.receipt_file.exists():
            print("没有发现{}.SHF的仓单文件,请先抓取数据!".format(self.quote_date_string))
            return
        with open(self.receipt_file, "r", encoding="utf-8") as reader:
            source_content = json.load(reader)
        json_df = pd.DataFrame(source_content['o_cursor'])
        # 处理仓库名称
        json_df["WHABBRNAME"] = json_df["WHABBRNAME"].apply(full_width_to_half_width)
        json_df["WHABBRNAME"] = json_df["WHABBRNAME"].apply(lambda name: name.split("$$")[0].strip())
        # 去掉含仓库名称含合计或小计的行
        json_df = json_df[~json_df['WHABBRNAME'].str.contains('总计|小计|合计')]  # 选取品种不含有小计和总计合计的行
        # 处理品种名称
        json_df["VARNAME"] = json_df["VARNAME"].apply(lambda name: name.split("$$")[0].strip())
        # 增加交易代码列
        json_df["VAREN"] = json_df['VARNAME'].apply(self.get_variety_en)
        # 仓单和增减转为int类型
        json_df["WRTWGHTS"] = json_df["WRTWGHTS"].apply(int)
        json_df["WRTCHANGE"] = json_df["WRTCHANGE"].apply(int)
        # 整理出想要的数据列(仓库名称(简称),交易代码,仓单,增减)
        result_df = json_df.reindex(columns=["WHABBRNAME", "VAREN", "WRTWGHTS", "WRTCHANGE"])
        result_df.columns = ["warehouse", "variety_en", "receipt", "increase"]
        # 增加ex_total字段 ex_total=0的计入小计
        result_df['ex_total'] = 0
        result_df['receipt_date'] = self.quote_date.strftime('%Y-%m-%d')
        receipt_data = result_df.to_dict(orient='records')
        print('----- 解析{}.SHF仓单文件2成功,数量:{} -----'.format(self.quote_date_string, len(receipt_data)))
        if len(receipt_data) > 1:
            # 保存到本地文件
            with open(self.resolution_receipt_file2, 'w', encoding='utf8') as fp:
                json.dump(receipt_data, fp, indent=2, ensure_ascii=False)

    @staticmethod
    def get_variety_en(variety_name: str):
        en = VARIETY_EN.get(variety_name.strip(), None)
        if not en:
            raise ValueError("品种:{} 的交易所代码配置不存在".format(variety_name))
        return en

    # 读取解析好的行情数据
    def read_daily_quote(self):
        if not self.resolution_quote_file.exists():
            raise ValueError('{}.SHF行情数据文件不存在,请先解析保存！'.format(self.quote_date_string))
        # 读取本地文件
        with open(self.resolution_quote_file, 'r', encoding='utf8') as fp:
            quote_data = json.load(fp)
        return quote_data

    # 读取解析好的排名数据
    def read_daily_rank(self):
        if not self.resolution_rank_file.exists():
            raise ValueError('{}.SHF排名数据文件不存在,请先解析保存！'.format(self.quote_date_string))
        # 读取本地文件
        with open(self.resolution_rank_file, 'r', encoding='utf8') as fp:
            rank_data = json.load(fp)
        return rank_data

    # 读取解析好的仓单数据
    def read_daily_receipt(self):
        if not self.resolution_receipt_file.exists():
            raise ValueError('{}.SHF仓单数据文件不存在,请先解析保存！'.format(self.quote_date_string))
        # 读取本地文件
        with open(self.resolution_receipt_file, 'r', encoding='utf8') as fp:
            receipt_data = json.load(fp)
        return receipt_data

    # 读取解析好的仓单数据2
    def read_daily_receipt2(self):
        if not self.resolution_receipt_file2.exists():
            raise ValueError('{}.SHF仓单数据文件2不存在,请先解析保存！'.format(self.quote_date_string))
        # 读取本地文件
        with open(self.resolution_receipt_file2, 'r', encoding='utf8') as fp:
            receipt_data = json.load(fp)
        return receipt_data

    # 保存日行情数据
    def save_daily_quote(self):
        quote_data = self.read_daily_quote()
        try:
            r = requests.post(self.quote_save_url, json=quote_data, verify=False)
            print(r.json())
        except Exception as e:
            print('保存{}.SHF行情数据失败了:{}'.format(self.quote_date_string, e))
        time.sleep(1)

    # 保存日排名数据
    def save_daily_rank(self):
        rank_data = self.read_daily_rank()
        try:
            r = requests.post(self.rank_save_url, json=rank_data, verify=False)
            print(r.json())
        except Exception as e:
            print('保存{}.SHF排名数据失败了:{}'.format(self.quote_date_string, e))
        time.sleep(1)

    # 保存仓单数据
    def save_daily_receipt(self):
        receipt_data = self.read_daily_receipt()
        if len(receipt_data) < 1:
            print('没有发现{}.SHF可以保存的仓单数据!'.format(self.quote_date_string))
            return
        try:
            r = requests.post(self.receipt_save_url, json=receipt_data, verify=False)
            print(r.json())
        except Exception as e:
            print('保存{}.SHF仓单数据失败了:{}'.format(self.quote_date_string, e))
        time.sleep(1)

    # 总表接口:保存行情数据，所有交易所数据放一个表的
    def new_save_quote(self):
        quote_data = self.read_daily_quote()
        df = pd.DataFrame(quote_data)
        df['quotes_date'] = df['date'].apply(lambda x: datetime.datetime.fromtimestamp(x).strftime('%Y-%m-%d'))
        df.rename(columns={'empty_volume': 'position_volume'}, inplace=True)

        save_url = self.SERVER_API + 'dat/quotes/daily-quotes/'
        try:
            r = requests.post(save_url, json=df.to_dict(orient='records'), verify=False)
            r_data = r.json()
        except Exception as e:
            print('新版保存{}.SHF行情数据失败了:{}'.format(self.quote_date_string, e))
        else:
            print('新版保存{}.SHF行情数据成功:{},message:{}'.format(self.quote_date_string, r_data['count'], r_data['message']))
        time.sleep(1)

    # 总表接口:保存排名数据，所有交易所数据放一个表的
    def new_save_rank(self):
        rank_data = self.read_daily_rank()
        df = pd.DataFrame(rank_data)
        df['rank_date'] = df['date'].apply(lambda x: datetime.datetime.fromtimestamp(x).strftime('%Y-%m-%d'))
        save_url = self.SERVER_API + 'dat/rank/daily-rank/'
        try:
            r = requests.post(save_url, json=df.to_dict(orient='records'), verify=False)
            r_data = r.json()
        except Exception as e:
            print('新版保存{}.SHF排名数据失败了:{}'.format(self.quote_date_string, e))
        else:
            print(
                '新版保存{}.SHF排名数据成功:{},message:{}'.format(self.quote_date_string, r_data['count'], r_data['message']))
        time.sleep(1)

    # 总表接口:保存仓单数据，所有交易所数据放一个表的
    def new_save_receipt(self):
        receipt_data = self.read_daily_receipt2()
        df = pd.DataFrame(receipt_data)
        save_url = self.SERVER_API + 'dat/receipt/daily-receipt/'
        try:
            r = requests.post(save_url, json=df.to_dict(orient='records'), verify=False)
            r_data = r.json()
        except Exception as e:
            print('新版保存{}.SHF仓单数据失败了:{}'.format(self.quote_date_string, e))
        else:
            print(
                '新版保存{}.SHF仓单数据成功:{},message:{}'.format(self.quote_date_string, r_data['count'], r_data['message']))
        time.sleep(1)

    def run_positions_handler(self):
        # 日行情
        quote_data = self.read_daily_quote()
        quotes_df = pd.DataFrame(quote_data)
        quotes_df.rename(columns={'date': 'quotes_ts', 'empty_volume': 'position_volume'}, inplace=True)
        # 日排名
        rank_data = self.read_daily_rank()
        rank_df = pd.DataFrame(rank_data)
        rank_df.rename(columns={'date': 'rank_ts'}, inplace=True)
        if quotes_df.empty or rank_df.empty:
            print('{}.CZC行情或持仓数据为空!'.format(self.quote_date_string))
            return
        quotes_df = quotes_df[['quotes_ts', 'variety_en', 'contract', 'close_price', 'trade_volume', 'position_volume']]
        rank_df = rank_df[['rank_ts', 'variety_en', 'contract', 'long_position', 'short_position']]
        dominant_df = get_dominant_price_position(quotes_df.copy(), rank_df.copy())
        # 获取合约的持仓数据
        contract_df = get_contract_price_position(quotes_df.copy(), rank_df.copy())
        final_df = pd.concat([dominant_df, contract_df])
        for col in ['close_price', 'position_price', 'position_volume', 'long_position', 'short_position']:
            final_df[col] = final_df[col].apply(lambda x: int(x) if int(x) == float(x) else round(x, 4))
        final_df.sort_values(by='contract', inplace=True)
        # for row in final_df.to_dict(orient='records'):
        #     print(row)
        # 保存到服务器
        save_url = self.SERVER_API + 'dsas/price-position/'
        try:
            r = requests.post(save_url, json=final_df.to_dict(orient='records'), verify=False)
            r_data = r.json()
        except Exception as e:
            print('新版保存{}.SHF持仓价格数据失败了:{}'.format(self.quote_date_string, e))
        else:
            print('新版保存{}.SHF持仓价格数据成功:{}'.format(self.quote_date_string, r_data['message']))


if __name__ == '__main__':
    SPIDER = 0
    PARSER = 1
    delta_days = 0
    d = datetime.datetime.today() + datetime.timedelta(days=delta_days)
    # d = datetime.datetime.strptime('20220303', '%Y%m%d')
    if SPIDER:
        spider = DailyQuoteSpider(date=d)
        spider.get_quote_file()  # 获取日行情数据保存为文件
        spider.get_rank_file()  # 获取日排名数据保存为文件
        spider.get_receipt_file()  # 获取日仓单数据保存为文件
    else:
        parser = DailyQuoteParser(date=d)
        if PARSER:
            parser.parse_quote_file()
            parser.parse_rank_file()
            parser.parse_receipt_file()
            parser.parse_receipt_file2()
        else:
            # --- 分库分表 ---
            parser.save_daily_quote()  # 保存日行情
            parser.save_daily_rank()  # 保存日排名
            parser.save_daily_receipt()  # 保存日仓单
            # --- 总表 ---
            parser.new_save_quote()  # 保存解析的行情数据
            parser.new_save_rank()  # 保存解析的持仓数据
            parser.new_save_receipt()  # 保存仓单的数据(含仓库简称)
            # -- 净持仓 ---
            parser.run_positions_handler()  # 处理保存净持仓数据
